package searcherPackage;

import java.util.HashSet;
import java.util.Set;
import java.util.StringTokenizer;
import java.util.TreeSet;

/**
 * @author i_khri
 */
public class QueryPreProcessor {

    /**
     * Tokenize theString and stem each token.
     */
    public static Set<String> extractTerms(String theString){
        Set<String> myDict = new TreeSet();


        StringTokenizer st = new StringTokenizer(theString, " ()");

        while (st.hasMoreTokens()) {

            // Getting rid of punctuation, stop words, and numbers
            String token = st.nextToken();
            token = token.replaceAll(".*[\\s\\p{Punct}].*", "");
            token = token.trim();
            token = token.replaceAll("[\\d]*", "");

            if(token.length()<2 || googleStopwords.contains(token)) {
                continue;
            }
            // Stemming the tokens
            char[] cToken = token.toLowerCase().toCharArray();
            Stemmer stemmer = new Stemmer();
            stemmer.add(cToken, cToken.length);
            stemmer.stem();
            token = stemmer.toString();

            // Adding token to the dictionary
            myDict.add(token);

        }
        return myDict;
    }


    public static String preProcessQuery(String query){
        // insert OR between every word
        StringBuilder q = new StringBuilder();
        StringTokenizer st = new StringTokenizer(query, " ");
        while(st.hasMoreTokens()){
            q.append(st.nextToken());
            if(!st.hasMoreTokens()){
                break;
            }
            q.append(" ");
            q.append("OR");
            q.append(" ");
            
        }
        return q.toString();
    }



    private static final Set<String> googleStopwords = new HashSet<String>();
    static {
        //googleStopwords.add("I".toUpperCase()); googleStopwords.add("a".toUpperCase());
        googleStopwords.add("about".toUpperCase()); googleStopwords.add("an".toUpperCase());
        googleStopwords.add("are".toUpperCase()); googleStopwords.add("as".toUpperCase());
        googleStopwords.add("at".toUpperCase()); googleStopwords.add("be".toUpperCase());
        googleStopwords.add("by".toUpperCase()); googleStopwords.add("com".toUpperCase());
        googleStopwords.add("de".toUpperCase()); googleStopwords.add("en".toUpperCase());
        googleStopwords.add("for".toUpperCase()); googleStopwords.add("from".toUpperCase());
        googleStopwords.add("how".toUpperCase()); googleStopwords.add("in".toUpperCase());
        googleStopwords.add("is".toUpperCase()); googleStopwords.add("it".toUpperCase());
        googleStopwords.add("la".toUpperCase()); googleStopwords.add("of".toUpperCase());
        googleStopwords.add("on".toUpperCase()); googleStopwords.add("or".toUpperCase());
        googleStopwords.add("that".toUpperCase()); googleStopwords.add("the".toUpperCase());
        googleStopwords.add("this".toUpperCase()); googleStopwords.add("to".toUpperCase());
        googleStopwords.add("was".toUpperCase()); googleStopwords.add("what".toUpperCase());
        googleStopwords.add("when".toUpperCase()); googleStopwords.add("where".toUpperCase());
        googleStopwords.add("who".toUpperCase()); googleStopwords.add("will".toUpperCase());
        googleStopwords.add("with".toUpperCase()); googleStopwords.add("and".toUpperCase());
        googleStopwords.add("the".toUpperCase()); googleStopwords.add("www".toUpperCase());
        googleStopwords.add("not".toUpperCase());
    }

}
